<?php 
	// 自动注册珍爱id
	function registerZhenaiIdbak()
	{
		//step1: get zhenai view cookie
		$url = "http://register.zhenai.com/register/register.jsps";
		$page = makeRequest($url,
						 "GET",
						 "",
						 true,
						 ROOT_PATH."/zhenai/cookie/zhenai_view_cookie.txt",
						 "",  
						 ""
						 ); 
		//step2: auto register
		$url = "http://register.zhenai.com/register/reg55OnView.jsps";
		$base = substr(strval(microtime(true)), -8);
		$parameter = array("snsType" => "",
							"whichTV" => "",
							"regType" => "mail",
							"hideMobile" => "",
							"viewMemberId" => "",
							"baseInfo.sex" => "0",
							"baseInfo.sex" => "0",
							"dateForm.year" => "1988",
							"dateForm.month" => "3",
							"dateForm.day" => "14",
							"areaForm.workProvince" => "10102000",
							"areaForm.workCity" => "10102001",
							"baseInfo.marriage" => "3",
							"baseInfo.education" => "3",
							"baseInfo.salary" => "9",
							"baseInfo.house" => "2",
							"baseInfo.children" => "1",
							"baseInfo.height" => "172",
							"loginInfo.email" => 'detect'.($base).'@126.com',
							"loginInfo.pwd" => "123abc",
							"baseInfo.nickname" => "kkkkkkkkkkk",
							"baseInfo2.servicemobile" => "13902340234",
							"baseInfo.introducecontent" => "%C7%EB%D3%C3%C4%FA%B5%C4%BB%B0%CF%F2%CE%D2%C3%C7%D0%CE%C8%DD%D2%BB%CF%C2%C4%FA%D7%D4%BC%BA%A3%AC%D4%D9%C3%E8%CA%F6%D2%BB%CF%C2%C4%FA%D0%C4%C4%BF%D6%D0%C0%ED%CF%EB%B5%C4%B0%E9%C2%C2%C7%EB%D3%C3%C4%FA%B5%C4%BB%B0%CF%F2%CE%D2%C3%C7%D0%CE%C8%DD%D2%BB%CF%C2%C4%FA%D7%D4%BC%BA%A3%AC%D4%D9%C3%E8%CA%F6%D2%BB%CF%C2%C4%FA%D0%C4%C4%BF%D6%D0%C0%ED%CF%EB%B5%C4%B0%E9%C2%C2%C7%EB%D3%C3%C4%FA%B5%C4%BB%B0%CF%F2%CE%D2%C3%C7%D0%CE%C8%DD%D2%BB%CF%C2%C4%FA%D7%D4%BC%BA%A3%AC%D4%D9%C3%E8%CA%F6%D2%BB%CF%C2%C4%FA%D0%C4%C4%BF%D6%D0%C0%ED%CF%EB%B5%C4%B0%E9%C2%C2%C7%EB%D3%C3%C4%FA%B5%C4%BB%B0%CF%F2%CE%D2%C3%C7%D0%CE%C8%DD%D2%BB%CF%C2%C4%FA%D7%D4%BC%BA%A3%AC%D4%D9%C3%E8%CA%F6%D2%BB%CF%C2%C4%FA%D0%C4%C4%BF%D6%D0%C0%ED%CF%EB%B5%C4%B0%E9%C2%C2%C7%EB%D3%C3%C4%FA%B5%C4%BB%B0%CF%F2%CE%D2%C3%C7%D0%CE%C8%DD%D2%BB%CF%C2%C4%FA%D7%D4%BC%BA%A3%AC%D4%D9%C3%E8%CA%F6%D2%BB%CF%C2%C4%FA%D0%C4%C4%BF%D6%D0%C0%ED%CF%EB%B5%C4%B0%E9%C2%C2%C7%EB%D3%C3%C4%FA%B5%C4%BB%B0%CF%F2%CE%D2%C3%C7%D0%CE%C8%DD%D2%BB%CF%C2%C4%FA%D7%D4%BC%BA%A3%AC%D4%D9%C3%E8%CA%F6%D2%BB%CF%C2%C4%FA%D0%C4%C4%BF%D6%D0%C0%ED%CF%EB%B5%C4%B0%E9%C2%C2%C7%EB%D3%C3%C4%FA%B5%C4%BB%B0%CF%F2%CE%D2%C3%C7%D0%CE%C8%DD%D2%BB%CF%C2%C4%FA%D7%D4%BC%BA%A3%AC%D4%D9%C3%E8%CA%F6%D2%BB%CF%C2%C4%FA%D0%C4%C4%BF%D6%D0%C0%ED%CF%EB%B5%C4%B0%E9%C2%C2",
							"xinmin" => "0",
							);
		$opt_arr = array(CURLOPT_REFERER => "http://register.zhenai.com/register/register.jsps",  //referer
						 CURLOPT_HEADER => true,						//输出中包含头信息		
					);
		printlog("using:".$parameter['loginInfo.email']." as register email", "register");
		$page = makeRequest($url,
						 "POST",
						 $parameter,
						 true, 
						 ROOT_PATH."/zhenai/cookie/zhenai_login_cookie.txt",  
						 ROOT_PATH."/zhenai/cookie/zhenai_view_cookie.txt",
						 $opt_arr
						 );
						 
		//step3: parse uid from page
		preg_match_all("/mailReceiverId=([0-9]+)&objectId/", $page, $arr, PREG_SET_ORDER);
		if (empty($arr)) {
			return false;
		}
		else{
			return $arr[0][1];
		}
	}
	// 自动注册珍爱id
	function registerZhenaiId()
	{
		//step1: get zhenai view cookie
		$url = "http://register.zhenai.com/register/register.jsps";
		$page = makeRequest($url,
						 "GET",
						 "",
						 true,
						 ROOT_PATH."/zhenai/cookie/zhenai_view_cookie.txt",
						 "",  
						 ""
						 ); 
		//step2: auto register
		$url = "http://register.zhenai.com/register/reg55OnView.jsps";
		$base = substr(strval(microtime(true)), -8);
		$parameter = array("snsType" => "",
							"isReliveCardMember" => -1,
							"copyTag" => "",
							"whichTV" => "",
							"regType" => "phone",
							"hideMobile" => "",
							"viewMemberId" => "",
							"baseInfo.sex" => "1",
							"dateForm.year" => "1988",
							"dateForm.month" => "3",
							"dateForm.day" => "14",
							"areaForm.workProvince" => "10102000",
							"areaForm.workCity" => "10102001",
							"baseInfo.marriage" => "3",
							"baseInfo.education" => "3",
							"baseInfo.salary" => "9",
							"baseInfo.house" => "2",
							"baseInfo.children" => "1",
							"baseInfo.height" => "172",
							"loginInfo.email" => 'detect'.($base).'@126.com',
							"baseInfo2.servicemobile" => 'detect'.($base).'@126.com',
							"loginInfo.pwd" => "123ffabc",
							"baseInfo.nickname" => $base,
							"baseInfo2.servicemobile" => makeRandMobile(),
							"xinmin" => "0",
							);
		$opt_arr = array(CURLOPT_REFERER => "http://register.zhenai.com/register/register.jsps",  //referer
						 CURLOPT_HEADER => true,						//输出中包含头信息		
					);
		printlog("using:".$parameter['loginInfo.email']." as register email", "register");
		$page = makeRequest($url,
						 "POST",
						 $parameter,
						 true, 
						 ROOT_PATH."/zhenai/cookie/zhenai_login_cookie.txt",  
						 ROOT_PATH."/zhenai/cookie/zhenai_view_cookie.txt",
						 $opt_arr
						 );
						 
		//step3: parse uid from page
		//printlog("----:".$page."----", "page");
		preg_match_all("/mailReceiverId=([0-9]+)&objectId/", $page, $arr, PREG_SET_ORDER);
		if (empty($arr)) {
			return false;
		}
		else{
			return $arr[0][1];
		}
	}
	function makeRandMobile(){
		$prefix = array('134','135','136','137','138','139','150','151','152','157','158','159','147','182','183','187','188','130','131','132','155','156','185','186','145','133','','153','','180','','189');
		$rand_keys = array_rand($prefix, 1);
		$mobile = $prefix[$rand_keys].createRandStr();
		return $mobile;
	}
	/*
	function createRandStr($n = 8)
	{	
		$s = '';
		$str = "0123456789";   //   输出字符集  
		$len = strlen($str)-1;
		for($i=0 ; $i<$n; $i++){
			$s .=  $str[rand(0,$len)];  
		}
		
		return $s;
	}*/
	//自动保存新注册的id
	function saveZhenaiId($new_zhenai_id)
	{
		global $conn;
		$today = date("Y-m-d", time());
		$now = date("Y-m-d H:i:s", time());
		$info = $now."\t".$new_zhenai_id."\n";
		//保存至文件（防止数据库挂掉，备查）
		file_put_contents(ROOT_PATH."/zhenai/log/register_log/$today.log", $info, FILE_APPEND);
		//保存至数据库
		$arr = array("site" => "zhenai",
					"ymd" => $today,
					"register_time" => $now,
					"source_user_id" => $new_zhenai_id
				);
		$sql = Sql::insert($arr, "register_log");
		$flag = $conn->query($sql);
		return $flag;
	}
	
	//获取某一天内珍爱最小的id
	function getZhenaiMinId($ymd)
	{
		global $conn;
		$sql = "select min(source_user_id) from register_log where ymd='$ymd' and site='zhenai'";
		$minId = $conn->result($sql);
		return $minId;
	}
	
	//获取某一天内珍爱最大的id
	function getZhenaiMaxId($ymd)
	{
		global $conn;
		$sql = "select max(source_user_id) from register_log where ymd='$ymd' and site='zhenai'";
		$maxId = $conn->result($sql);
		return $maxId;
	}
	
	//获取某一天内珍爱已经抓到的最大id
	function getZhenaiMaxIdFinished($ymd)
	{
		global $conn;
		$tblName = getTblName($ymd);
		$sql = "select max(source_user_id) from $tblName where fetch_date='$ymd' and `from`='zhenai'";//echo $sql;exit;
		$maxIdFinished = $conn->result($sql);
		return $maxIdFinished;
	}
	
	//获取珍爱某一个id的注册时间，debug时候用
	function getZhenaiRegTime($uid)
	{
		global $conn;
		$sql = "select register_time from register_log where source_user_id='$uid' and site='zhenai'";
		$regTime = $conn->result($sql);
		return $regTime;
	}
	
	//判断珍爱抓下来的页面是否包含正常的uid（通过经验判断，审核不通过的uid页面会非常小）
	function checkZhenaiIdExists($page)
	{
		if(strlen($page)<130)
			return false;
		else
			return true;
	}
	
	//主函数： 根据起始id，终止id 和 ymd 这几个值来处理珍爱业务，包括抓取、提取和入库
	function zhenai_crawl($startId, $endId, $ymd)
	{
		//创建page目录
		$page_dir = ROOT_PATH."/zhenai/pages";
		if(!is_dir($page_dir)){
			mkdir($page_dir);
		}
		
		$page_dir = $page_dir."/".$ymd."/";
		if(!is_dir($page_dir)){
			mkdir($page_dir);
		}
		/*
		$continueMiss = 0; //连续id不存在的个数
		for ($uid=$startId; $uid<$endId; $uid++){					
			$page = "";	
			//避免断线情况，这里采用循环抓取，直到抓到页面为止
			do{
				$url = "http://album.zhenai.com/profile/getmemberdata.jsps?memberid=".$uid;
				//printlog("start crawl url : $url");
				$page = makeRequest($url, 
								"GET" , 
								"", 
								false, 
								"", 
								ROOT_PATH."/zhenai/cookie/zhenai_request_cookie.txt",
								""
								);
			} while($page === false);

			if(checkzhenaiIdExists($page)){
				printlog("$uid available", "crawl");
				if(file_put_contents($page_dir.$uid.".html", $page)){
					doWithZhenaiPage($uid, $ymd, $page);
				}
				else{
					printlog("save page error", "crawl");
				}
				$continueMiss = 0;
			}else{
				printlog("$uid not available", "crawl");
				$continueMiss ++ ;
			}
	
			//防止出现id跳跃的现象，如果连续50个不存在，则以间隔50向前跳，直到抓取到一个id为止再恢复间隔为1
			if($continueMiss > 50){
				$uid += 50;
			}
		}
		*/
		//如果抓取的速度不够，改用multi
		$number = 30;//concurrent request number
		$continueMiss = 0;
		$previousExists = true;
		$previousId = $startId;
		for ($uid=$startId; $uid<$endId; ){
			//保留本次开始的uid，万一抓取失败了重新抓取
			$previousId = $uid; 
			$perRequestFailedCount  = 0 ;//记录本次抓取失败的个数
			$handles = makeZhenaiMultiRequests($uid, $number);
			foreach($handles as $thisuid => $handle){
				$page = curl_multi_getcontent($handle);
				if($page === false){ 
					printlog ("curl_multi_getcontent error: ".curl_error($handle), "crawl");
				}
				
				if(checkZhenaiIdExists($page)){
					printlog("$thisuid available", "crawl");
					if(file_put_contents($page_dir.$thisuid.".html", $page)){	
						doWithzhenaiPage($thisuid, $ymd, $page);
					}
					else{
						printlog("save page error", "crawl");
					}
					$continueMiss = 0;
				}else{
					if(strlen($page)<30){
						echo "network problem ret is ".$page;
						$perRequestFailedCount++;
					}
					else{ 
						printlog("$thisuid not available", "crawl");
					}
					$continueMiss ++;
				}
				curl_close($handle);
				$uid++;
			}
				
			if($perRequestFailedCount>= $number/2){//如果有一半不存在，说明中途网络有问题了，这个条件只针对zhenai.com有效
				$uid = $previousId;
				printlog("本次并发有 $perRequestFailedCount 个抓取失败，重新来。。。", "crawl");
				continue;//重新来
			}
			else{
				printlog("本次并发有 $perRequestFailedCount 个抓取失败，继续下一次抓取", "crawl");
			}
				
			//防止出现id跳跃的现象，如果连续50个不存在，则以间隔50向前跳，直到抓取到一个id为止再恢复间隔为1
			if($continueMiss > 50){
			//	$uid += 50;
			}	
		}
		
	
	} 

	function makeZhenaiMultiRequests($uid, $number)
	{
		$mhandle = curl_multi_init();
		$handles = array();
		while($number>0){
			$url = "http://album.zhenai.com/profile/getmemberdata.jsps?memberid=".$uid;
			$opt_arr = array(CURLOPT_REFERER => "http://search.baidu.com/search",  //referer	
					);
			$handles[$uid] = makeRequestResource($url,
												"GET" , 
												"", 
												false, 
												"", 
												ROOT_PATH."/zhenai/cookie/zhenai_request_cookie.txt", 
												$opt_arr
												);
			$uid++;
			$number--;
		}
		foreach($handles as $handle)
			curl_multi_add_handle($mhandle, $handle);
		//wait all request finish
		do {
			$status = curl_multi_exec($mhandle, $active);
		} while ($status === CURLM_CALL_MULTI_PERFORM || $active);
		curl_multi_close($mhandle);
		
		return $handles;
	}
	
	//生成珍爱登录后的cookie
	function makeZhenaiCookie()
	{
		//step1: 
		/*makeRequest("http://album.zhenai.com/profile/getmemberdata.jsps?memberid=4721963", 
					"GET", 
					"", 
					true, 
					ROOT_PATH."/zhenai/zhenai_userhome_cookie.txt", 
					"",
					""
					);
		
		//step2:
		makeRequest("http://register.zhenai.com/register/register.jsps?sometimes=true&viewMemberId=4721963", 
					"GET", 
					"", 
					true, 
					ROOT_PATH."/zhenai/zhenai_userregister_cookie.txt", 
					ROOT_PATH."/zhenai/zhenai_request_cookie.txt",
					""
					);
		
		//step3: combile cookies
		file_put_contents('zhenai_request_cookie.txt', file_get_contents('zhenai_userregister_cookie.txt'), FILE_APPEND);
		*/
		makeRequest("http://register.zhenai.com/register/register.jsps?sometimes=true&viewMemberId=4721963", 
					"GET", 
					"", 
					true, 
					ROOT_PATH."/zhenai/cookie/zhenai_request_cookie.txt", 
					"",
					""
					);
	}
		
	//对一个抓下的珍爱页面的后续处理
	function doWithZhenaiPage($uid, $ymd, $page)
	{
		//check uid exists?
		global $conn, $global_province_list;;
		$tblName = getTblName();
		$sql = "select * from $tblName where `from`='zhenai' and source_user_id='$uid'";
		$exists = $conn->has_record($sql);
		if ($exists){
			printlog("$uid exists, do nothing", "crawl");
			return 1;
		}
		else{
			//extract
			$page = iconv("GBK", 'UTF-8//IGNORE', $page);
			//tmp: 23岁，北京东城区，男士
			$tmp = simpleExtract('%<liclass="intro">(.*?)</li>%i', $page, "tmp");
			//printlog($tmp);exit;
			$province = "";
			foreach ($global_province_list as $prov){
				if (strpos($tmp, $prov) !== false) {
					$province = $prov; //北京
					break;
				}
			}

			$gender = (strpos($tmp, "男士") !== false ) ? 1 : 0; //男为0 女为1


			$age = simpleExtract('/(\d\d)岁/', $tmp, "age"); //23
			
			$city = simpleExtract('%住在<strongclass="c_01">(.*?)</strong>%i', $tmp, "tmp");

			$city = str_replace(array($province, "省"), "", $city); //东城区

			//printlog ("age: $age province: $province city: $city gender: $gender tmp: $tmp ");exit;
			$has_photo = simpleExtract('%<ulid="showArea"class="clearfix">(.*)</ul>%i', $page, "has_photo");
			$has_photo = ($has_photo)?1:0;

			
			//insert
			$arr = array("fetch_date" => $ymd,
					"`from`" => "zhenai",
					"source_user_id" => $uid,
					"age" => $age,
					"province" => $province,
					"city" => $city,
					"gender" => $gender,
					"has_photo" => $has_photo,
				);
		
			$sql = Sql::insert($arr, $tblName);
			//$conn->query($sql);
		}
	}
	
	

?>